import csv
import os
import re
import time

import requests
from lxml import etree


class youxin:
    def __init__(self):

        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 FS',
            'Cookie': 'pif=; RELEASE_KEY=; XIN_anti_uid=BBAD3517-F2B7-201A-A057-832F6FC67979; XIN_LOCATION_CITY=%7B%22cityid%22%3A%221520%22%2C%22cityname%22%3A%22%5Cu5bbf%5Cu8fc1%22%2C%22ename%22%3A%22suqian%22%2C%22service%22%3A%221%22%2C%22zhigou%22%3A%221%22%2C%22is_visit%22%3A%221%22%2C%22city_rank%22%3A%22100%22%2C%22is_gold_partner%22%3A%22-1%22%2C%22direct_rent_support%22%3A%221%22%2C%22is_wz_mortgage%22%3A%221%22%7D; uid=CvQuyF/Re+aBNwASE722Ag==; SEO_SOURCE=https://gitee.com/rong_cun_qing/python19_exam_spider_month/blob/master/%E7%AC%AC02%E9%A2%98.md; SEO_REF=https://gitee.com/rong_cun_qing/python19_exam_spider_month/blob/master/%E7%AC%AC02%E9%A2%98.md; XIN_UID_CK=d1aa18da-b010-88af-503c-eb22201196af; Hm_lvt_ae57612a280420ca44598b857c8a9712=1607564270; session_xin=mdmhroiot6l8mmv8bltr8m6ger37ukss; acw_tc=249c4ea516075690948054291e0797044d0bd523bf72477db9702a24be; acw_sc__v2=5fd1900dee6bc9901f416bfc415779dae4fbdfa0; Hm_lpvt_ae57612a280420ca44598b857c8a9712=1607569426; SERVERID=42a40ad1d64e7196e0d380af3dbf8a5f|1607569425|1607564262'
        }
        self.download_path = os.getcwd() + '/优信二手车/'
        if not os.path.exists(self.download_path):
            os.mkdir(self.download_path)

    def get_info(self):
        for i in range(1, 6):
            url = 'https://www.xin.com/suqian/i%s/' % str(i)
            resp = requests.get(url=url, headers=self.headers).text
            html = etree.HTML(resp)
            li_list = html.xpath('//div[@class="_list-con list-con clearfix ab_carlist"]/ul/li')
            now_page_car_list = []
            for li in li_list:
                car_dict = {}
                img_url_list = li.xpath('./div[@class="across"]/a/img/@data-original')

                if not img_url_list:
                    img_url = li.xpath('./div[@class="across"]/a/img/@src')[0]
                    if img_url[:4] != 'http':
                        img_url = 'https:' + img_url
                else:
                    img_url = img_url_list[0]
                    if img_url[:4] != 'http':
                        img_url = 'https:' + img_url

                title = li.xpath('./div[@class="across"]/a/@title')[0]

                price = li.xpath('./div[@class="across"]/a/div/p/em/text()')[0]
                price = price.replace(' ', '').replace('\n', '').replace('\r', '').replace('\t', '').replace('万', '')

                year = li.xpath('./div[@class="across"]/a/div/span/text()')[0]
                year = year.replace(' ', '').replace('\n', '').replace('\r', '').replace('\t', '').replace('年', '')

                now_time = int(time.time())
                # 转换为其他日期格式,如:"%Y-%m-%d %H:%M:%S"
                timeArray = time.localtime(now_time)
                spider_time = time.strftime("%Y-%m-%d", timeArray)

                car_dict['title'] = title
                car_dict['price'] = price
                car_dict['year'] = year
                car_dict['spider_time'] = spider_time
                car_dict['img_url'] = img_url
                now_page_car_list.append(car_dict)
                # break
            self.save_csv(now_page_car_list)
            self.save_img(now_page_car_list)
            print('第%s页数据下载完成' % str(i))

    def save_img(self, now_page_car_list):
        for car_info in now_page_car_list:
            ret = re.match(r'(.*?) (.*?) ', car_info.get('title'))
            brand = ret.group(1)
            car_type = ret.group(2)
            if not os.path.exists(self.download_path + brand + '/images'):
                os.mkdir(self.download_path + brand + '/images')
            img_url = car_info.get('img_url')
            img_type = img_url[-4:]
            img_content = requests.get(url=img_url).content
            with open(self.download_path + brand + '/images/' + car_info.get('title') + img_type, 'wb') as f1:
                f1.write(img_content)
            print('%s的图片下载完成' % car_type)

    def save_csv(self, now_page_car_list):
        for car_info in now_page_car_list:

            ret = re.match(r'(.*?) (.*?) ', car_info.get('title'))
            brand = ret.group(1)
            car_type = ret.group(2)
            if not os.path.exists(self.download_path + brand):
                os.mkdir(self.download_path + brand)

            with open(self.download_path + brand + '/' + car_type + '.csv', 'a', newline='') as f1:
                f_csv = csv.DictWriter(f1, ['title', 'price', 'year', 'spider_time', 'img_url'])
                f_csv.writerow(car_info)


if __name__ == '__main__':
    ershouche = youxin()
    ershouche.get_info()